import json
from io import BytesIO

from re_common.baselibrary.utils.core.mdeprecated import try_except2_async, try_except2, retry_func_async

from apps.crawler_platform.core_callback import *
from apps.crawler_platform.core_callback.err_callback import all_client_back_callback
from apps.crawler_platform.core_platform.core_nosql import NoSqlUtil
from apps.crawler_platform.core_platform.core_sql import CoreSqlMixin
from apps.crawler_platform.core_platform.g_model import EtlDealModel, \
    EtlCallBackModel, ModelColl, DownModel
from apps.crawler_platform.etl_platform.base_etl_step import BaseETLStep, BaseEtlItems

# 不执行逻辑，目的是为了不让 from apps.crawler_platform.core_callback import *
# 被编辑器认为没有使用，快捷清除时就不会被清除
from apps.crawler_platform.util.nosqlhelper import NOSQLHelper
from apps.sql_app.mmongodb import Coll

STATICA_HACK = False
# globals()['kcah_acitats'[::-1].upper()] = False
if STATICA_HACK:  # pragma: no cover
    null()


class EtlItems(BaseEtlItems):

    def __init__(self):
        super(EtlItems, self).__init__()
        # id 数量不定，不能定死，用一个字典接收键值对
        self.id_dicts: dict = {}
        self.field: dict = {}
        self.down_model_x = None
        self.deal_model: EtlDealModel = EtlDealModel()
        self.mongo_html: dict = {}

    def get_pkm(self):
        """
        获取主键
        :return:
        """
        pkm = {"task_name": self.task_name,
               "task_tag": self.task_tag
               }
        pkm.update(self.id_dicts)
        return pkm


class EtlParse(BaseETLStep):

    def __init__(self, request, inputs):
        super().__init__()
        self.err_tag = "etl_parse"
        self.pm = EtlItems()

        self.input = inputs
        self.pm.task_name = inputs.data.task_name
        self.pm.task_tag = inputs.data.task_tag
        self.pm.id_dicts = inputs.data.id_dicts
        self.pm.field = {"last_data": 1}
        self.pm.Id = "_".join([v for k, v in self.pm.id_dicts.items()])

        self.pm.redis = request.app.state.redis

        self.request = request
        self.step_info = ""
        # 解析字段字典
        self.para_dicts = {}

        self.sql_model_class = None
        self.html_model_class = None

    def set_model(self):
        self.sql_model_class = ModelColl.get_sql_model(self.pm.task_setting.sql_model_name)
        self.html_model_class = ModelColl.get_html_model(self.pm.task_setting.model_name)
        self.pm.down_model = DownModel[self.html_model_class](down_dict={})
        self.pm.down_model_x = self.html_model_class

    async def set_journal_info(self, row):
        if row:
            article_info_json = row.get("article_info_json", "")
            if len(article_info_json) > 0:
                jjson = json.loads(article_info_json)
                journal_rawid = jjson.get("journal_rawid", "")
                if row['task_name'] in ['sciencedirectjournal', 'cambridgejournal', 'pnasjournal', 'frontiersinjournal']:
                    return row
                if len(journal_rawid) > 0:
                    j_row = await CoreSqlMixin.get_sql_etl_parse_journal_info(self.pm.task_name,
                                                                              journal_rawid)
                    row["journal_info"] = json.loads(j_row["journal_json"])
        return row

    async def set_other_journal_info(self, row):
        if row:
            article_json = row.get("article_json", "")
            if len(article_json) == 0:
                article_json = row.get("article_info_json", "")
            if len(article_json) > 0:
                jjson = json.loads(article_json)
                journal_rawid = jjson.get("journal_raw_id", "")
                if len(journal_rawid) > 0:
                    j_row = await CoreSqlMixin.get_sql_etl_parse_other_journal_info(self.pm.task_name, self.pm.task_tag,
                                                                                    journal_rawid)
                    row["journal_info"] = json.loads(j_row["list_json"])
        return row

    @try_except2_async(callback=all_client_back_callback, is_print=True)
    async def get_sql(self):
        self.set_model()
        # row = await CoreSqlMixin.get_qk_etl_local_article( # 后面会废弃这个类
        row = await CoreSqlMixin.get_sql_etl_parse(
            self.pm.task_setting.table_name,
            self.pm.task_setting.table_index,
            self.pm.task_name,
            self.pm.task_tag,
            self.pm.id_dicts)

        if not row:
            return False, {
                "err_msg": "qk article local etl sql查询不存在;" + ",".join([self.pm.task_tag, str(self.pm.id_dicts)])}

        if self.pm.task_setting.sql_model_name == "JournalEtlArticleModel":
            # 如果使用这个类需要特殊处理
            row = await self.set_journal_info(row)
        elif self.pm.task_setting.sql_model_name in ["OtherArticleEtlModel", "OtherArticleSpeEtlModel"]:
            row = await self.set_other_journal_info(row)
        self.pm.sql_model = self.sql_model_class.parse_obj(row)
        return True, row
        #
        # if row and row.get(self.pm.task_setting.final_stat, "") in [1, 7, 8, 9]:
        #     self.pm.sql_model = self.sql_model_class.parse_obj(row)
        #     return True, row
        # else:
        #     if not row:
        #         return False, {
        #             "err_msg": "qk article local etl sql查询不存在;" + ",".join([self.pm.task_tag, str(self.pm.id_dicts)])}
        #     else:
        #         return False, {"code": row["is_true"],
        #                        "err_msg": "该记录不在解析状态里"}

    @try_except2()
    def verify_para(self):
        """
        验证解析出来的数据
        """
        return True, {}

    @try_except2_async(callback=all_client_back_callback, is_print=True)
    async def save_sql(self, code, err_msg=""):
        """
        与关系型sql的处理 包括存储下一级的sql以及本级sql的状态和字段更新
        """
        if code is False:
            # 如果前面的布尔值是fasle，就直接写7错误
            code = 7
            err_msg = str(err_msg)
        else:
            if self.pm.deal_model.status == "FAILED":
                code = self.pm.deal_model.code
                err_msg = self.pm.deal_model.err_msg
            else:
                code = 9
                err_msg = ""
        table_name = self.pm.task_setting.table_name
        table_index = self.pm.task_setting.table_index
        stname = self.pm.task_setting.final_stat

        print(table_name, table_index, stname, self.pm.task_name,
              self.pm.task_tag, self.pm.id_dicts["rawid"], code,
              err_msg)

        await CoreSqlMixin.update_etl_local_article(table_name, table_index, stname, self.pm.task_name,
                                                    self.pm.task_tag, self.pm.id_dicts["rawid"], code,
                                                    err_msg)
        return True, ""

    @try_except2_async(callback=all_client_back_callback, is_print=True)
    async def deal_html_etl(self):
        self.step_info = self.add_all_process_info(f"处理解析，通过配置css和xpath进行解析")
        bools, err_info = await self.para_next()
        if not bools:
            return bools, err_info
        # if "status" in self.para_dicts.keys() and self.para_dicts["status"] == "FAILED":
        #     return False, {"err_msg": json.dumps(self.para_dicts, ensure_ascii=False)}
        # from app_main import app
        # self.para_dicts["journal_rawids"] = await app.state.redis.hgetall("journal_rawids")
        self.para_dicts["journal_rawids"] = self.pm.etl_redis_journal_rawids
        self.para_dicts["redis"] = self.pm.redis
        callmodel = EtlCallBackModel[
            self.sql_model_class,
            self.html_model_class](
            down_model=self.pm.down_model,
            para_dicts=self.para_dicts,
            sql_model=self.pm.sql_model,
            redis_all=self.redis_model
        )
        # self.pm.deal_model is EtlDealModel
        self.pm.deal_model = self.callback_deal_html_etl(callmodel)
        # if self.pm.deal_model.status == "FAILED":
        #     return False, {"err_msg": self.pm.deal_model.err_msg}
        ref_state = self.pm.deal_model.ref_state
        table_latest = json.loads(self.pm.task_setting.table_latest)
        if len(ref_state) > 0:
            mongo_conn = Coll.get_table_conn(table_latest["ref"])  # 根据table名取得mongo链接
            cjip = await NOSQLHelper.find_one(mongo_conn, {"_id": ref_state["lngid"]})
            if cjip:
                cjip["is_deprecated"] = "1"
                self.pm.deal_model.save_data.append({"table": table_latest["ref"], "data": cjip})
                ref_depr_conn = Coll.get_table_conn("ref_deprecated")
                await NOSQLHelper.insert_one(ref_depr_conn, {"lngid": cjip["lngid"], "data": cjip})
        if "image" in self.pm.deal_model.other_info.keys():
            # print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
            other_info = self.pm.deal_model.other_info
            save_path = other_info["image"]["path"]
            image_data = other_info["image"]["image_data"]
            bytesIO = BytesIO(image_data)
            from app_main import app
            from settings import get_settings
            sets = get_settings()
            async with app.state.aioboto3.client('s3', endpoint_url=sets.AWS_LOCAL_ENDPOINT_URL) as s3:
                await s3.upload_fileobj(bytesIO, sets.AWS_LOCAL_BUCKET, save_path)
                chk_one = await s3.head_object(Bucket=sets.AWS_LOCAL_BUCKET, Key=save_path)
                if chk_one is None:
                    raise Exception("保存图片到minio失败")

        return True, {}

    def callback_deal_html_etl(self, callmodel) -> EtlDealModel:
        name = "_".join([self.pm.task_name, self.pm.task_tag, "etl", "callback"])
        callback = eval(name)
        return callback(callmodel)

    @retry_func_async(retry_times=3, sleep_time=2)
    @try_except2_async(callback=all_client_back_callback, is_print=True)
    async def read_nosql_html(self):
        """
        存储到nosql数据库
        """
        # 期刊列表级默认保存历史
        bools, dicts = await NoSqlUtil.read_nosql_html(self.pm)
        # 这里是针对cnkiimage的逻辑
        if not bools:
            return bools, dicts
        data = self.pm.mongo_html["data"]
        if not data or not data[0]["last_data"]:
            self.pm.one_para.task_set.nosql_table = "cnkiimage_html"
            bools, dicts = await NoSqlUtil.read_nosql_html(self.pm)
            if not bools:
                return bools, dicts
        data = self.pm.mongo_html["data"]
        # 如果mongo查出来不存在
        if not data or not data[0]["last_data"]:
            return False, {
                "err_msg": "qk article local etl mongoDB里journal_article未查询到数据;" + str(self.pm.get_pkm())}
        data = data[0]["last_data"]
        try:
            del data["down_date"]
            del data["down_time"]
        except:
            pass
        self.pm.down_model = self.pm.down_model.set_parse_obj(data)
        return True, {}
